{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "title部分使用 bigru；content 部分使用cnn.\n",
    "\n",
    "两部分分别使用不同的 embedding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting ...\n",
      "Prepared, costed time 0.0644391 s.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt \n",
    "import pickle\n",
    "import os\n",
    "import sys\n",
    "import shutil\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "time0 = time.time()\n",
    "print('Starting ...')\n",
    "model_name = 'ch5-1-2embed-rnn256-cnn2345'                    # 模型名称\n",
    "W_embedding = np.load('/home1/huangyongye/mygithub/zhihu/data/ch_W_embedding.npy').astype(np.float32)            # 导入预训练好的词向量\n",
    "model_path = '../ckpt/' + model_name + '/'                  # 模型保存位置\n",
    "summary_path = '../summary/' + model_name + '/'             # summary 位置\n",
    "result_path = '../result/' + model_name + '.csv'            # result.csv 位置\n",
    "scores_path = '../scores/' + model_name + '.npy'            # scores.npy 位置\n",
    "\n",
    "if not os.path.exists(model_path):\n",
    "    os.makedirs(model_path)         \n",
    "model_path = model_path + 'model.ckpt'\n",
    "if os.path.exists(summary_path):   # 删除原来的 summary 文件，避免重合\n",
    "    shutil.rmtree(summary_path)\n",
    "os.makedirs(summary_path)          # 然后再次创建\n",
    "\n",
    "\n",
    "# ##################### config ######################\n",
    "n_step1 = max_len1 = 52           # 句子长度\n",
    "n_step2 = max_len2 = 300\n",
    "input_size = embedding_size = 256       # 字向量长度\n",
    "\n",
    "# title-bigru\n",
    "hidden_size = 256    # 隐含层节点数\n",
    "n_layer = 1        # bi-gru 层数\n",
    "# content-cnn\n",
    "filter_sizes = [2,3,4,5]                  # 卷积核大小\n",
    "n_filter = 256                          # 每种卷积核的个数\n",
    "n_filter_total = n_filter * len(filter_sizes)\n",
    "# output\n",
    "fc_hidden_size = 1024                   # fc 层节点数\n",
    "n_class = 1999\n",
    "\n",
    "global_step = 0\n",
    "print('Prepared, costed time %g s.' % (time.time() - time0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building model ...\n",
      "Finished creating the bi-gru model.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "from tensorflow.contrib import rnn\n",
    "import tensorflow.contrib.layers as layers\n",
    "\n",
    "'''\n",
    "双端 GRU，知乎问题多标签分类。\n",
    "'''\n",
    "\n",
    "print('Building model ...')\n",
    "lr = tf.placeholder(tf.float32)\n",
    "keep_prob = tf.placeholder(tf.float32, [])\n",
    "batch_size = tf.placeholder(tf.int32, [])  # 注意类型必须为 tf.int32\n",
    "tst = tf.placeholder(tf.bool)\n",
    "n_updates = tf.placeholder(tf.int32)      # training iteration,传入 bn 层\n",
    "update_emas = list()   # BN 层中所有的更新操作\n",
    "\n",
    "\n",
    "def weight_variable(shape, name, initializer=None):\n",
    "    \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n",
    "    if initializer is None:\n",
    "        initializer = tf.contrib.layers.xavier_initializer()\n",
    "    return tf.get_variable(name=name, shape=shape, initializer=initializer, dtype=tf.float32)\n",
    "\n",
    "def bias_variable(shape, name):\n",
    "    \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    return tf.get_variable(name=name, initializer=initial, dtype=tf.float32)\n",
    "\n",
    "\n",
    "def batchnorm(Ylogits, is_test, num_updates, offset, convolutional=False):\n",
    "    \"\"\"batchnormalization.\n",
    "    Args:\n",
    "        Ylogits: 1D向量或者是3D的卷积结果。\n",
    "        num_updates: 迭代的global_step\n",
    "        offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。\n",
    "        scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。\n",
    "        m: 表示batch均值；v:表示batch方差。\n",
    "        bnepsilon：一个很小的浮点数，防止除以 0.\n",
    "    Returns:\n",
    "        Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。\n",
    "        update_moving_everages：更新mean和variance，主要是给最后的 test 使用。\n",
    "    \"\"\"\n",
    "    exp_moving_avg = tf.train.ExponentialMovingAverage(0.999, num_updates) # adding the iteration prevents from averaging across non-existing iterations\n",
    "    bnepsilon = 1e-5\n",
    "    if convolutional:\n",
    "        mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n",
    "    else:\n",
    "        mean, variance = tf.nn.moments(Ylogits, [0])\n",
    "    update_moving_everages = exp_moving_avg.apply([mean, variance])\n",
    "    m = tf.cond(is_test, lambda: exp_moving_avg.average(mean), lambda: mean)\n",
    "    v = tf.cond(is_test, lambda: exp_moving_avg.average(variance), lambda: variance)\n",
    "    Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n",
    "    return Ybn, update_moving_everages\n",
    "\n",
    "# 第一次定义，放在 CPU 上面\n",
    "with tf.device('/gpu:0'):\n",
    "    with tf.variable_scope('title_embed'):\n",
    "        title_embedding = tf.get_variable(name=\"title_embedding\", shape=W_embedding.shape, \n",
    "                        initializer=tf.constant_initializer(W_embedding), trainable=True)   # fine-tune\n",
    "    with tf.variable_scope('content_embed'):\n",
    "        content_embedding = tf.get_variable(name=\"content_embedding\", shape=W_embedding.shape, \n",
    "                        initializer=tf.constant_initializer(W_embedding), trainable=True)   # fine-tune\n",
    "        \n",
    "with tf.name_scope('Inputs'):\n",
    "    X1_inputs = tf.placeholder(tf.int64, [None, n_step1], name='X1_input')\n",
    "    X2_inputs = tf.placeholder(tf.int64, [None, n_step2], name='X2_input')\n",
    "    y_inputs = tf.placeholder(tf.float32, [None, n_class], name='y_input')\n",
    "    inputs1 = tf.nn.embedding_lookup(title_embedding, X1_inputs)\n",
    "    inputs2 = tf.nn.embedding_lookup(content_embedding, X2_inputs)\n",
    "\n",
    "def gru_cell():\n",
    "    with tf.name_scope('gru_cell'):\n",
    "        cell = rnn.GRUCell(hidden_size, reuse=tf.get_variable_scope().reuse)\n",
    "    return rnn.DropoutWrapper(cell, output_keep_prob=keep_prob)\n",
    "\n",
    "\n",
    "def bi_gru(inputs):\n",
    "    \"\"\"build the bi-GRU network. 返回个最后一层的隐含状态。\"\"\"      \n",
    "    cells_fw = [gru_cell() for _ in range(n_layer)]\n",
    "    cells_bw = [gru_cell() for _ in range(n_layer)]\n",
    "    initial_states_fw = [cell_fw.zero_state(batch_size, tf.float32) for cell_fw in cells_fw]\n",
    "    initial_states_bw = [cell_bw.zero_state(batch_size, tf.float32) for cell_bw in cells_bw] \n",
    "    outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs, \n",
    "                        initial_states_fw = initial_states_fw, initial_states_bw = initial_states_bw, dtype=tf.float32)\n",
    "    return outputs\n",
    "    \n",
    "\n",
    "def task_specific_attention(inputs, output_size,\n",
    "                            initializer=layers.xavier_initializer(),\n",
    "                            activation_fn=tf.tanh, scope=None):\n",
    "    \"\"\"\n",
    "    Performs task-specific attention reduction, using learned\n",
    "    attention context vector (constant within task of interest).\n",
    "    Args:\n",
    "        inputs: Tensor of shape [batch_size, units, input_size]\n",
    "            `input_size` must be static (known)\n",
    "            `units` axis will be attended over (reduced from output)\n",
    "            `batch_size` will be preserved\n",
    "        output_size: Size of output's inner (feature) dimension\n",
    "    Returns:\n",
    "        outputs: Tensor of shape [batch_size, output_dim].\n",
    "    \"\"\"\n",
    "    assert len(inputs.get_shape()) == 3 and inputs.get_shape()[-1].value is not None\n",
    "    with tf.variable_scope(scope or 'attention') as scope:\n",
    "        # u_w, attention 向量\n",
    "        attention_context_vector = tf.get_variable(name='attention_context_vector',\n",
    "                                                   shape=[output_size],\n",
    "                                                   initializer=initializer,\n",
    "                                                   dtype=tf.float32)\n",
    "        # 全连接层，把 h_i 转为 u_i ， shape= [batch_size, units, input_size] -> [batch_size, units, output_size]\n",
    "        input_projection = layers.fully_connected(inputs, output_size,\n",
    "                                                  activation_fn=activation_fn,\n",
    "                                                  scope=scope)\n",
    "        # 输出 [batch_size, units]\n",
    "        vector_attn = tf.reduce_sum(tf.multiply(input_projection, attention_context_vector), axis=2, keep_dims=True)\n",
    "        attention_weights = tf.nn.softmax(vector_attn, dim=1)   \n",
    "        tf.summary.histogram('attention_weigths', attention_weights)\n",
    "        weighted_projection = tf.multiply(inputs, attention_weights)   # ???源代码,原文应该是对 hi 加权！！\n",
    "        outputs = tf.reduce_sum(weighted_projection, axis=1)\n",
    "        return outputs   # 输出 [batch_size, hidden_size*2]\n",
    "\n",
    "\n",
    "def textcnn(inputs, n_step):\n",
    "    \"\"\"build the TextCNN network. Return the h_drop\"\"\"\n",
    "    # inputs.shape = [batchsize, n_step, embedding_size]\n",
    "    inputs = tf.expand_dims(inputs, -1)\n",
    "    pooled_outputs = list()\n",
    "    for i, filter_size in enumerate(filter_sizes):\n",
    "        with tf.variable_scope(\"conv-maxpool-%s\" % filter_size):\n",
    "            # Convolution Layer\n",
    "            filter_shape = [filter_size, embedding_size, 1, n_filter]\n",
    "            W_filter = weight_variable(shape=filter_shape, name='W_filter')\n",
    "            beta = bias_variable(shape=[n_filter], name='beta_filter')\n",
    "            tf.summary.histogram('beta_filter', beta)\n",
    "            conv = tf.nn.conv2d(inputs, W_filter, strides=[1, 1, 1, 1], padding=\"VALID\", name=\"conv\")\n",
    "            conv_bn, update_ema = batchnorm(conv, tst, n_updates, beta, convolutional=True)    # 在激活层前面加 BN\n",
    "            # Apply nonlinearity, batch norm scaling is not useful with relus\n",
    "            # batch norm offsets are used instead of biases,使用 BN 层的 offset，不要 biases\n",
    "            h = tf.nn.relu(conv_bn, name=\"filter_relu\")\n",
    "            # Maxpooling over the outputs\n",
    "            pooled = tf.nn.max_pool(h,ksize=[1, n_step - filter_size + 1, 1, 1],\n",
    "                strides=[1, 1, 1, 1],padding='VALID',name=\"pool\")\n",
    "            pooled_outputs.append(pooled)\n",
    "            update_emas.append(update_ema)\n",
    "    h_pool = tf.concat(pooled_outputs, 3)\n",
    "    h_pool_flat = tf.reshape(h_pool, [-1, n_filter_total]) \n",
    "    return h_pool_flat    # shape = [-1, n_filter_total]\n",
    "\n",
    "\n",
    "# title-bigru 部分\n",
    "with tf.variable_scope('bigru_title'):          # 两部分的输出都加上 BN+RELU 层，\n",
    "    word_encoder_title = bi_gru(inputs1)      # title 部分输出\n",
    "    output_title = task_specific_attention(word_encoder_title, hidden_size*2)\n",
    "    \n",
    "# content-cnn 部分    \n",
    "with tf.variable_scope('cnn-content'):\n",
    "    output_content = textcnn(inputs2, n_step2)\n",
    "    \n",
    "\n",
    "with tf.name_scope('fc_layer'): # 将两个网络的输出拼接后加一个 fc 层\n",
    "    output = tf.concat([output_title, output_content], axis=1)\n",
    "    W_fc = weight_variable([hidden_size*2+n_filter_total, fc_hidden_size], name='Weight_fc')\n",
    "    tf.summary.histogram('W_fc', W_fc)\n",
    "    h_fc = tf.matmul(output, W_fc, name='h_fc')\n",
    "    beta_fc = tf.Variable(tf.constant(0.1, tf.float32, shape=[fc_hidden_size], name=\"beta_fc\"))\n",
    "    tf.summary.histogram('beta_fc', beta_fc)\n",
    "    fc_bn, update_ema_fc = batchnorm(h_fc, tst, n_updates, beta_fc, convolutional=False)\n",
    "    update_emas.append(update_ema_fc)\n",
    "    fc_bn_relu = tf.nn.relu(fc_bn, name=\"relu\")\n",
    "    \n",
    "    \n",
    "with tf.name_scope('out_layer'):\n",
    "    W_out = weight_variable([fc_hidden_size, n_class], name='Weight_out') \n",
    "    tf.summary.histogram('Weight_out', W_out)\n",
    "    b_out = bias_variable([n_class], name='bias_out') \n",
    "    tf.summary.histogram('bias_out', b_out)\n",
    "    y_pred = tf.nn.xw_plus_b(fc_bn_relu, W_out, b_out, name='y_pred')  #每个类别的分数 scores\n",
    "\n",
    "with tf.name_scope('cost'):\n",
    "    cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=y_pred, labels=y_inputs))\n",
    "    tf.summary.scalar('cost', cost)\n",
    "    \n",
    "# ***** 优化求解 *******\n",
    "with tf.name_scope('AdamOptimizer1'):\n",
    "    tvars1 = tf.trainable_variables()\n",
    "    grads1 = tf.gradients(cost, tvars1)\n",
    "    optimizer1 = tf.train.AdamOptimizer(learning_rate=lr)\n",
    "    train_op1 = optimizer1.apply_gradients(zip(grads1, tvars1),\n",
    "        global_step=tf.contrib.framework.get_or_create_global_step())\n",
    "    \n",
    "with tf.name_scope('AdamOptimizer2'):\n",
    "    tvars2 = [tvar for tvar in tvars1 if 'title_embed' not in tvar.name]\n",
    "    grads2 = tf.gradients(cost, tvars2)\n",
    "    optimizer2 = tf.train.AdamOptimizer(learning_rate=lr)\n",
    "    train_op2 = optimizer2.apply_gradients(zip(grads2, tvars2),\n",
    "        global_step=tf.contrib.framework.get_or_create_global_step())\n",
    "update_op = tf.group(*update_emas)   # 更新 BN 参数\n",
    "   \n",
    "# summary\n",
    "merged = tf.summary.merge_all() # summary\n",
    "train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n",
    "test_writer = tf.summary.FileWriter(summary_path + 'test')\n",
    "print 'Finished creating the bi-gru model.'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_tr_batch=22656\n",
      "n_va_batches=782\n",
      "Every thing prepared!\n"
     ]
    }
   ],
   "source": [
    "sys.path.append('..')\n",
    "from data_helpers import BatchGenerator\n",
    "from data_helpers import to_categorical\n",
    "from evaluator import score_eval\n",
    "\n",
    "\n",
    "data_train_path = '/home2/huangyongye/zhihu/ch-data/data_train/'\n",
    "data_valid_path = '/home2/huangyongye/zhihu/ch-data/data_valid/'\n",
    "tr_batches = os.listdir(data_train_path)   # batch 文件名列表\n",
    "va_batches = os.listdir(data_valid_path)\n",
    "n_tr_batches = len(tr_batches)\n",
    "n_va_batches = len(va_batches)\n",
    "\n",
    "\n",
    "# 测试用\n",
    "# n_tr_batches = 2000  \n",
    "# n_va_batches = 100\n",
    "print('n_tr_batch=%d' % n_tr_batches)\n",
    "print('n_va_batches=%d' % n_va_batches)\n",
    "\n",
    "\n",
    "def get_batch(data_path, batch_id, title_len=n_step1):\n",
    "    \"\"\"get a batch from data_path\"\"\"\n",
    "    new_batch = np.load(data_path + str(batch_id) + '.npz')\n",
    "    X_batch = new_batch['X']\n",
    "    y_batch = new_batch['y']\n",
    "    X1_batch = X_batch[:, :title_len]\n",
    "    X2_batch = X_batch[:, title_len:]\n",
    "    return [X1_batch, X2_batch, y_batch]\n",
    "\n",
    "\n",
    "def valid_epoch(data_path=data_valid_path):\n",
    "    \"\"\"Test on the valid data.\"\"\"\n",
    "    _costs = 0.0\n",
    "    predict_labels_list = list()  # 所有的预测结果\n",
    "    marked_labels_list = list()   # 真实标签\n",
    "    for i in xrange(n_va_batches):\n",
    "        [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n",
    "        marked_labels_list.extend(y_batch)\n",
    "        y_batch = to_categorical(y_batch)\n",
    "        _batch_size = len(y_batch)\n",
    "        fetches = [merged, cost, y_pred]  \n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch,  y_inputs:y_batch, lr:_lr,\n",
    "                     batch_size:_batch_size, keep_prob:1.0, tst:True, n_updates:global_step}\n",
    "        summary, _cost, predict_labels = sess.run(fetches, feed_dict)\n",
    "        _costs += _cost\n",
    "        predict_labels = map(lambda label: label.argsort()[-1:-6:-1], predict_labels) # 取最大的5个下标\n",
    "        predict_labels_list.extend(predict_labels)\n",
    "    predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n",
    "    precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n",
    "    mean_cost = _costs / n_va_batches\n",
    "    return mean_cost, precision, recall, f1\n",
    "\n",
    "print('Every thing prepared!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_tr_batches=22656\n"
     ]
    }
   ],
   "source": [
    "# 测试\n",
    "# max_epoch = 2\n",
    "# max_max_epoch = 3 \n",
    "# decay_step = 1000\n",
    "# valid_step = 500\n",
    "\n",
    "# 正式 \n",
    "max_epoch = 2\n",
    "max_max_epoch = 7 \n",
    "decay_step = 10000\n",
    "valid_step = 8000\n",
    "\n",
    "\n",
    "_lr = 6e-4\n",
    "decay1 = 0.95\n",
    "decay2 = 0.65\n",
    "print('n_tr_batches=%d' % n_tr_batches)\n",
    "saver = tf.train.Saver(max_to_keep=3)           # 最多保存的模型数量\n",
    "sess.run(tf.global_variables_initializer())\n",
    "last_f1 = 0.39\n",
    "model_num = 0\n",
    "global_step = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH 1, lr= 0.0006\n",
      "Global_step=7999: valid cost=0.00684308; p=0.95088, r=0.396432, f1=0.279786, time=3799 s\n",
      "Global_step=15999: valid cost=0.00684718; p=1.06351, r=0.443074, f1=0.31277, time=4113.22 s\n",
      "EPOCH 2, lr= 0.0005415\n",
      "Global_step=23999: valid cost=0.00558799; p=1.18583, r=0.489811, f1=0.346633, time=3994.27 s\n",
      "Global_step=31999: valid cost=0.00552809; p=1.21437, r=0.501467, f1=0.354909, time=3990.56 s\n",
      "Global_step=39999: valid cost=0.00496429; p=1.25306, r=0.51511, f1=0.365046, time=3773.38 s\n",
      "('the save path is ', '../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-1')\n",
      "Begin updating embedding.\n",
      "EPOCH 3, lr= 0.000488704\n",
      "Global_step=47999: valid cost=0.00440273; p=1.2881, r=0.527851, f1=0.374419, time=4497.35 s\n",
      "Global_step=55999: valid cost=0.00421609; p=1.32505, r=0.542662, f1=0.384993, time=3528.16 s\n",
      "Global_step=63999: valid cost=0.00408048; p=1.3522, r=0.553103, f1=0.392539, time=3368.21 s\n",
      "('the save path is ', '../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-2')\n",
      "EPOCH 4, lr= 0.000206477\n",
      "Global_step=71999: valid cost=0.00403955; p=1.36269, r=0.557426, f1=0.3956, time=3363.14 s\n",
      "('the save path is ', '../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-3')\n",
      "Global_step=79999: valid cost=0.00403626; p=1.3655, r=0.558508, f1=0.396382, time=3339.84 s\n",
      "('the save path is ', '../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-4')\n",
      "Global_step=87999: valid cost=0.00399239; p=1.37095, r=0.560457, f1=0.397824, time=3332.97 s\n",
      "('the save path is ', '../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-5')\n",
      "EPOCH 5, lr= 5.67038e-05\n",
      "Global_step=95999: valid cost=0.00399456; p=1.3737, r=0.561731, f1=0.398697, time=3336.95 s\n",
      "('the save path is ', '../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-6')\n",
      "Global_step=103999: valid cost=0.00398935; p=1.37518, r=0.56221, f1=0.399062, time=3337.46 s\n",
      "('the save path is ', '../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-7')\n",
      "Global_step=111999: valid cost=0.00398814; p=1.3762, r=0.562736, f1=0.399414, time=3340.02 s\n",
      "('the save path is ', '../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-8')\n",
      "EPOCH 6, lr= 2.39574e-05\n",
      "Global_step=119999: valid cost=0.00399934; p=1.37526, r=0.561911, f1=0.398918, time=3327.88 s\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-be4b13e3299e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mglobal_step\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mglobal_step\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mvalid_step\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# 进行 valid 计算\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m             \u001b[0mvalid_cost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprecision\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecall\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalid_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m             print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n\u001b[1;32m     21\u001b[0m                     global_step, valid_cost, precision, recall, f1, time.time()-time0))\n",
      "\u001b[0;32m<ipython-input-3-c4e8d8dc1c45>\u001b[0m in \u001b[0;36mvalid_epoch\u001b[0;34m(data_path)\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0mmarked_labels_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m   \u001b[0;31m# 真实标签\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_va_batches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m         \u001b[0;34m[\u001b[0m\u001b[0mX1_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX2_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_batch\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m         \u001b[0mmarked_labels_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0my_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_categorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-3-c4e8d8dc1c45>\u001b[0m in \u001b[0;36mget_batch\u001b[0;34m(data_path, batch_id, title_len)\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0mnew_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_id\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'.npz'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0mX_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'X'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m     \u001b[0my_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'y'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m     \u001b[0mX1_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0mtitle_len\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m     \u001b[0mX2_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle_len\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/numpy/lib/npyio.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    231\u001b[0m                 return format.read_array(bytes,\n\u001b[1;32m    232\u001b[0m                                          \u001b[0mallow_pickle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mallow_pickle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 233\u001b[0;31m                                          pickle_kwargs=self.pickle_kwargs)\n\u001b[0m\u001b[1;32m    234\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/numpy/lib/format.pyc\u001b[0m in \u001b[0;36mread_array\u001b[0;34m(fp, allow_pickle, pickle_kwargs)\u001b[0m\n\u001b[1;32m    638\u001b[0m             \u001b[0mpickle_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    639\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 640\u001b[0;31m             \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpickle_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    641\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mUnicodeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    642\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversion_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/zipfile.pyc\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, n)\u001b[0m\n\u001b[1;32m    630\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    631\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 632\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    633\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    634\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mbuf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/zipfile.pyc\u001b[0m in \u001b[0;36mread1\u001b[0;34m(self, n)\u001b[0m\n\u001b[1;32m    647\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mBadZipfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Bad CRC-32 for file %r\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 649\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0mread1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    650\u001b[0m         \u001b[0;34m\"\"\"Read up to n bytes with at most one read() system call.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    651\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "time0 = time.time()\n",
    "train_op = train_op2 \n",
    "decay = decay1\n",
    "\n",
    "for epoch in xrange(max_max_epoch):\n",
    "    batch_indexs = np.random.permutation(n_tr_batches)  # shuffle the training data\n",
    "    if epoch == max_epoch:\n",
    "        train_op = train_op1\n",
    "        decay = decay2   # 降低学习率衰减\n",
    "        if model_num == 0:\n",
    "            model_num += 1\n",
    "            save_path = saver.save(sess, model_path, global_step=model_num)\n",
    "            print('the save path is ', save_path)\n",
    "        print('Begin updating embedding.')\n",
    "    print 'EPOCH %d, lr= %g' % (epoch+1, _lr)    \n",
    "    for batch in xrange(n_tr_batches): \n",
    "        global_step += 1\n",
    "        if (global_step+1) % valid_step == 0:    # 进行 valid 计算\n",
    "            valid_cost, precision, recall, f1 = valid_epoch()\n",
    "            print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n",
    "                    global_step, valid_cost, precision, recall, f1, time.time()-time0))\n",
    "            time0 = time.time()\n",
    "            if (f1 > last_f1):\n",
    "                last_f1 = f1\n",
    "                model_num += 1\n",
    "                save_path = saver.save(sess, model_path, global_step=model_num)\n",
    "                print('the save path is ', save_path) \n",
    "        if (global_step+1) % decay_step == 0:\n",
    "            _lr = _lr*decay\n",
    "                \n",
    "        batch_id = batch_indexs[batch]\n",
    "        [X1_batch, X2_batch, y_batch] = get_batch(data_train_path, batch_id, n_step1)\n",
    "        y_batch = to_categorical(y_batch)\n",
    "        _batch_size = len(y_batch)\n",
    "        fetches = [merged, cost, train_op, update_op]\n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch, y_inputs:y_batch, batch_size:_batch_size, lr:_lr, \n",
    "                     keep_prob:0.5, tst:False, n_updates:global_step}\n",
    "        summary, _cost, _, _ = sess.run(fetches, feed_dict) # the cost is the mean cost of one batch\n",
    "        if global_step % 100:\n",
    "            train_writer.add_summary(summary, global_step)\n",
    "            batch_id = np.random.randint(0, n_va_batches)   # 随机选一个验证batch\n",
    "            [X1_batch, X2_batch, y_batch] = get_batch(data_valid_path, batch_id, n_step1)\n",
    "            y_batch = to_categorical(y_batch)\n",
    "            _batch_size = len(y_batch)\n",
    "            feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch,  y_inputs:y_batch, lr:_lr,\n",
    "                         batch_size:_batch_size, keep_prob:1.0, tst:True, n_updates:global_step}\n",
    "            fetches = [merged, cost]\n",
    "            summary, _cost = sess.run(fetches, feed_dict)\n",
    "            test_writer.add_summary(summary, global_step)\n",
    "\n",
    "valid_cost, precision, recall, f1 = valid_epoch()  # # 每个 epoch 进行一次验证 valid\n",
    "print('Global_step=%d;  valid cost=%g; p=%g, r=%g, f1=%g; speed=%g s/epoch' % (\n",
    "    global_step, valid_cost, precision, recall, f1, time.time()-time0) )\n",
    "if (f1 > last_f1):\n",
    "    model_num += 1\n",
    "    save_path = saver.save(sess, model_path, global_step=model_num)\n",
    "    print('the save path is ', save_path) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 若没有收敛，继续迭代"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 本地测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ../ckpt/ch5-1-2embed-rnn256-cnn2345/model.ckpt-8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 1/782 [00:00<01:58,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "local predicting ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 782/782 [01:27<00:00,  8.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "local valid p=1.3762, r=0.562736, f1=0.399414; speed=87.7753 s/epoch\n",
      "Writed the scores into ../local_scores/ch5-1-2embed-rnn256-cnn2345.npy, time 88.3074 s\n"
     ]
    }
   ],
   "source": [
    "# 导入保存好的模型\n",
    "saver = tf.train.Saver()\n",
    "best_model_path = model_path + '-' +str(8)\n",
    "saver.restore(sess, best_model_path)\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "local_scores_path = '../local_scores/' + model_name + '.npy'\n",
    "\n",
    "# 导入测试数据\n",
    "def local_predict(scores_path=local_scores_path, data_path=data_valid_path):\n",
    "    \"\"\"预测  valid 结果，并保存预测概率 到  scores.csv 文件中。\"\"\"\n",
    "    print('local predicting ...')\n",
    "    time0 = time.time()\n",
    "    fetches = [y_pred]   \n",
    "    predict_labels_list = list()  # 所有的预测结果\n",
    "    marked_labels_list = list()   # 真实标签\n",
    "    predict_scores = list()\n",
    "    for i in tqdm(xrange(n_va_batches)):\n",
    "        [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n",
    "        marked_labels_list.extend(y_batch)\n",
    "        y_batch = to_categorical(y_batch)\n",
    "        _batch_size = len(y_batch)\n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch, y_inputs:y_batch,\n",
    "                     batch_size:_batch_size, keep_prob:1.0, tst:True, n_updates:global_step}\n",
    "        predict_labels = sess.run(fetches, feed_dict)[0]\n",
    "        predict_scores.append(predict_labels)\n",
    "        predict_labels = map(lambda label: label.argsort()[-1:-6:-1], predict_labels) # 取最大的5个下标\n",
    "        predict_labels_list.extend(predict_labels)\n",
    "    predict_scores = np.asarray(predict_scores)\n",
    "    predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n",
    "    precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n",
    "    print('local valid p=%g, r=%g, f1=%g; speed=%g s/epoch' % ( precision, recall, f1, time.time()-time0) )\n",
    "    np.save(local_scores_path, predict_scores)\n",
    "    print('Writed the scores into %s, time %g s' % (local_scores_path, time.time()-time0))\n",
    "    \n",
    "local_predict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对测试数据进行预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_path = '../data/'\n",
    "\n",
    "# 导入测试数据\n",
    "def predict():\n",
    "    \"\"\"预测测试集结果，并保存到  result.csv 文件中。\"\"\"\n",
    "    data_path = '../ch-data/data_test//'\n",
    "    n_te_batches = len(os.listdir(data_path))\n",
    "    fetches = [y_pred]   \n",
    "    predict_labels_list = list()  # 所有的预测结果\n",
    "    predict_scores = list()\n",
    "    for i in tqdm(xrange(n_te_batches)):\n",
    "        X_batch = np.load(data_path + str(i) + '.npy')\n",
    "        X1_batch = X_batch[:, :n_step1]\n",
    "        X2_batch = X_batch[:, n_step1:]\n",
    "        _batch_size = len(X_batch)\n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch,\n",
    "                     batch_size:_batch_size, keep_prob:1.0, tst:True, n_updates:global_step}\n",
    "        predict_labels = sess.run(fetches, feed_dict)[0]\n",
    "        predict_scores.append(predict_labels)\n",
    "        predict_labels = map(lambda label: label.argsort()[-1:-6:-1], predict_labels) # 取最大的5个下标\n",
    "        predict_labels_list.extend(predict_labels)\n",
    "    predict_scores = np.asarray(predict_scores)\n",
    "    return predict_labels_list, np.vstack(predict_scores)\n",
    "    \n",
    "def write_result(predict_labels_list, result_path):\n",
    "    \"\"\"把结果写到 result.csv 中\"\"\"\n",
    "    eval_question = np.load('../data/eval_question.npy')\n",
    "    with open('../data/sr_topic2id.pkl', 'rb') as inp:\n",
    "        sr_topic2id = pickle.load(inp)\n",
    "        sr_id2topic = pickle.load(inp)\n",
    "    pred_labels = np.asarray(predict_labels_list).reshape([-1])\n",
    "    pred_topics = sr_id2topic[pred_labels].values.reshape([-1, 5])   # 转为 topic\n",
    "    df_result = pd.DataFrame({'question':eval_question, 'tid0': pred_topics[:,0], 'tid1':pred_topics[:, 1],\n",
    "                         'tid2': pred_topics[:,2], 'tid3':pred_topics[:,3],'tid4': pred_topics[:,4]})\n",
    "    df_result.to_csv(result_path, index=False, header=False)\n",
    "    print('Finished writing the result')\n",
    "    return df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/1699 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting the result, writing into ../result/ch5-1-2embed-rnn256-cnn2345.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1699/1699 [02:58<00:00,  9.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished writing the result\n",
      "217360\n",
      "Saving the predict_scores into ../scores/ch5-1-2embed-rnn256-cnn2345.npy\n",
      "(217360, 1999)\n",
      "Finished! costed time 187.122 s\n"
     ]
    }
   ],
   "source": [
    "# 预测\n",
    "time0 = time.time()\n",
    "print('Predicting the result, writing into %s' % result_path)\n",
    "predict_labels_list,predict_scores = predict()\n",
    "df_result = write_result(predict_labels_list, result_path=result_path) \n",
    "print(len(df_result))  # 结果应该为 217360\n",
    "print('Saving the predict_scores into %s' % scores_path)\n",
    "print(predict_scores.shape)\n",
    "np.save(scores_path, predict_scores)\n",
    "print('Finished! costed time %g s' % (time.time()-time0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(217360, 1999)\n",
      "(1999,)\n",
      "[ -5.6614542   -7.4593277   -7.8017478  ..., -13.84312534 -14.89337063\n",
      "  -5.60859346]\n"
     ]
    }
   ],
   "source": [
    "print(np.vstack(predict_scores).shape)\n",
    "print(predict_scores[0].shape)\n",
    "print(predict_scores[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>tid0</th>\n",
       "      <th>tid1</th>\n",
       "      <th>tid2</th>\n",
       "      <th>tid3</th>\n",
       "      <th>tid4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6215603645409872328</td>\n",
       "      <td>-7506384235581390893</td>\n",
       "      <td>2919247920214845195</td>\n",
       "      <td>4610596224687453206</td>\n",
       "      <td>-6839713564940654454</td>\n",
       "      <td>-5932391056759866388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6649324930261961840</td>\n",
       "      <td>2858911571784840089</td>\n",
       "      <td>3418451812342379591</td>\n",
       "      <td>-240041917918953337</td>\n",
       "      <td>-7483543763655495143</td>\n",
       "      <td>3383016985780045156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-4251899610700378615</td>\n",
       "      <td>2919247920214845195</td>\n",
       "      <td>2816249700493135244</td>\n",
       "      <td>-7358589937244777363</td>\n",
       "      <td>-5265476641576484497</td>\n",
       "      <td>-5932391056759866388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6213817087034420233</td>\n",
       "      <td>-8655945395761165989</td>\n",
       "      <td>-4966205278807386328</td>\n",
       "      <td>5804619920623030604</td>\n",
       "      <td>-3966324279534387421</td>\n",
       "      <td>-3250879589215178639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-8930652370334418373</td>\n",
       "      <td>3972493657017129406</td>\n",
       "      <td>-8963554618409314978</td>\n",
       "      <td>-1115593437686158905</td>\n",
       "      <td>6018641953300645757</td>\n",
       "      <td>-7790543634407640064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-2893445476547593888</td>\n",
       "      <td>-6206436693745657677</td>\n",
       "      <td>2131451573312950491</td>\n",
       "      <td>-8274522839089381384</td>\n",
       "      <td>-8655945395761165989</td>\n",
       "      <td>-2696736445927423374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2614833994648160978</td>\n",
       "      <td>4482402820945758152</td>\n",
       "      <td>-6748914495015758455</td>\n",
       "      <td>-3856154743789177934</td>\n",
       "      <td>1553849747662134917</td>\n",
       "      <td>-3904210248050890128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1572988006266661060</td>\n",
       "      <td>3738968195649774859</td>\n",
       "      <td>-8932546057542867495</td>\n",
       "      <td>2904932941037075699</td>\n",
       "      <td>-578156641762363345</td>\n",
       "      <td>-8314147124186495753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-3736249911643942320</td>\n",
       "      <td>-7653703019053330516</td>\n",
       "      <td>8690965822342756180</td>\n",
       "      <td>1127459907694805235</td>\n",
       "      <td>5502528845814007324</td>\n",
       "      <td>-5932391056759866388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-976507019126932319</td>\n",
       "      <td>3195914392210930723</td>\n",
       "      <td>3804601920633030746</td>\n",
       "      <td>6940355838132160535</td>\n",
       "      <td>8313816860478517392</td>\n",
       "      <td>1757481363245650831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5515590371171945660</td>\n",
       "      <td>4195795391451929480</td>\n",
       "      <td>4351331710881888756</td>\n",
       "      <td>-1689319711084901730</td>\n",
       "      <td>3804601920633030746</td>\n",
       "      <td>8313816860478517392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>6888753259805649782</td>\n",
       "      <td>4931893522763265934</td>\n",
       "      <td>6810657854754527644</td>\n",
       "      <td>-6254806834008898035</td>\n",
       "      <td>3257681557352812938</td>\n",
       "      <td>2787171473654490487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>7873230180977758006</td>\n",
       "      <td>4730384719025183341</td>\n",
       "      <td>-3026760652624195547</td>\n",
       "      <td>-6440461292041887516</td>\n",
       "      <td>8461136052903019092</td>\n",
       "      <td>-5932391056759866388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1805154204046808321</td>\n",
       "      <td>3738968195649774859</td>\n",
       "      <td>-839691564858654120</td>\n",
       "      <td>-2627298052801704596</td>\n",
       "      <td>1160326435131345730</td>\n",
       "      <td>-8932546057542867495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>5466502219528315935</td>\n",
       "      <td>-3333307402327293632</td>\n",
       "      <td>5370870069386720811</td>\n",
       "      <td>2787171473654490487</td>\n",
       "      <td>5732944515355059947</td>\n",
       "      <td>616649108700697054</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                question                  tid0                  tid1  \\\n",
       "0    6215603645409872328  -7506384235581390893   2919247920214845195   \n",
       "1    6649324930261961840   2858911571784840089   3418451812342379591   \n",
       "2   -4251899610700378615   2919247920214845195   2816249700493135244   \n",
       "3    6213817087034420233  -8655945395761165989  -4966205278807386328   \n",
       "4   -8930652370334418373   3972493657017129406  -8963554618409314978   \n",
       "5   -2893445476547593888  -6206436693745657677   2131451573312950491   \n",
       "6    2614833994648160978   4482402820945758152  -6748914495015758455   \n",
       "7    1572988006266661060   3738968195649774859  -8932546057542867495   \n",
       "8   -3736249911643942320  -7653703019053330516   8690965822342756180   \n",
       "9    -976507019126932319   3195914392210930723   3804601920633030746   \n",
       "10   5515590371171945660   4195795391451929480   4351331710881888756   \n",
       "11   6888753259805649782   4931893522763265934   6810657854754527644   \n",
       "12   7873230180977758006   4730384719025183341  -3026760652624195547   \n",
       "13   1805154204046808321   3738968195649774859   -839691564858654120   \n",
       "14   5466502219528315935  -3333307402327293632   5370870069386720811   \n",
       "\n",
       "                    tid2                  tid3                  tid4  \n",
       "0    4610596224687453206  -6839713564940654454  -5932391056759866388  \n",
       "1    -240041917918953337  -7483543763655495143   3383016985780045156  \n",
       "2   -7358589937244777363  -5265476641576484497  -5932391056759866388  \n",
       "3    5804619920623030604  -3966324279534387421  -3250879589215178639  \n",
       "4   -1115593437686158905   6018641953300645757  -7790543634407640064  \n",
       "5   -8274522839089381384  -8655945395761165989  -2696736445927423374  \n",
       "6   -3856154743789177934   1553849747662134917  -3904210248050890128  \n",
       "7    2904932941037075699   -578156641762363345  -8314147124186495753  \n",
       "8    1127459907694805235   5502528845814007324  -5932391056759866388  \n",
       "9    6940355838132160535   8313816860478517392   1757481363245650831  \n",
       "10  -1689319711084901730   3804601920633030746   8313816860478517392  \n",
       "11  -6254806834008898035   3257681557352812938   2787171473654490487  \n",
       "12  -6440461292041887516   8461136052903019092  -5932391056759866388  \n",
       "13  -2627298052801704596   1160326435131345730  -8932546057542867495  \n",
       "14   2787171473654490487   5732944515355059947    616649108700697054  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_result.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>tid0</th>\n",
       "      <th>tid1</th>\n",
       "      <th>tid2</th>\n",
       "      <th>tid3</th>\n",
       "      <th>tid4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6215603645409872328</td>\n",
       "      <td>4610596224687453206</td>\n",
       "      <td>-6839713564940654454</td>\n",
       "      <td>5818382893362053755</td>\n",
       "      <td>-7506384235581390893</td>\n",
       "      <td>-5932391056759866388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6649324930261961840</td>\n",
       "      <td>3418451812342379591</td>\n",
       "      <td>2858911571784840089</td>\n",
       "      <td>-240041917918953337</td>\n",
       "      <td>2382911985227044227</td>\n",
       "      <td>-7483543763655495143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-4251899610700378615</td>\n",
       "      <td>2919247920214845195</td>\n",
       "      <td>-7358589937244777363</td>\n",
       "      <td>2816249700493135244</td>\n",
       "      <td>-7270992690764838239</td>\n",
       "      <td>-3315241959305847628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6213817087034420233</td>\n",
       "      <td>5804619920623030604</td>\n",
       "      <td>-4966205278807386328</td>\n",
       "      <td>-8655945395761165989</td>\n",
       "      <td>244937959911721367</td>\n",
       "      <td>7476760589625268543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-8930652370334418373</td>\n",
       "      <td>3972493657017129406</td>\n",
       "      <td>-8963554618409314978</td>\n",
       "      <td>-1115593437686158905</td>\n",
       "      <td>-6925670792665757873</td>\n",
       "      <td>6018641953300645757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               question                 tid0                  tid1  \\\n",
       "0   6215603645409872328  4610596224687453206  -6839713564940654454   \n",
       "1   6649324930261961840  3418451812342379591   2858911571784840089   \n",
       "2  -4251899610700378615  2919247920214845195  -7358589937244777363   \n",
       "3   6213817087034420233  5804619920623030604  -4966205278807386328   \n",
       "4  -8930652370334418373  3972493657017129406  -8963554618409314978   \n",
       "\n",
       "                   tid2                  tid3                  tid4  \n",
       "0   5818382893362053755  -7506384235581390893  -5932391056759866388  \n",
       "1   -240041917918953337   2382911985227044227  -7483543763655495143  \n",
       "2   2816249700493135244  -7270992690764838239  -3315241959305847628  \n",
       "3  -8655945395761165989    244937959911721367   7476760589625268543  \n",
       "4  -1115593437686158905  -6925670792665757873   6018641953300645757  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 参考结果\n",
    "df_result.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在全部预测正确的情况下，理论值为：f1=0.713933\n",
    "precision=2.50273, recall=0.998873, f1=0.713933"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "在全部预测正确的情况下，理论值为：\n",
      "precision=2.50273, recall=0.998873, f1=0.713933\n"
     ]
    }
   ],
   "source": [
    "# 假设全部正确，f1 值最高能到多少\n",
    "def padding_label(labels):\n",
    "    \"\"\"把所有的label补齐到长度为 5\"\"\"\n",
    "    label_len = len(labels)\n",
    "    if label_len >= 5:\n",
    "        return labels[:5]\n",
    "    return np.hstack([labels, np.zeros(5-label_len, dtype=int) - 1])\n",
    "    \n",
    "\n",
    "marked_labels_list = data_valid.y.tolist() # 所有的标注结果\n",
    "predict_labels_list = map(padding_label, marked_labels_list)\n",
    "predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n",
    "precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n",
    "print '在全部预测正确的情况下，理论值为：'\n",
    "print 'precision=%g, recall=%g, f1=%g' % (precision, recall, f1)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
